{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "\n",
    "SPAM_PATH = os.path.join(\"datasets\",\"spam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "HAM_DIR = os.path.join(SPAM_PATH,\"easy_ham\")\n",
    "SPAM_DIR = os.path.join(SPAM_PATH,\"spam\")\n",
    "# 分别得到所有的邮件名\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name) > 20]            # 正常邮件名称\n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name) > 20]         # 垃圾邮件名称"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 我们可以使用python的 \"email\"模块解析这些电子邮件（它处理邮件头，编码等）\n",
    "import email \n",
    "import email.policy\n",
    "# 定义一个读取邮件的方法，一次只能读取一个邮件\n",
    "def load_email(is_spam,filename,spam_path=SPAM_PATH):\n",
    "    directory = \"spam\" if is_spam else \"easy_ham\"\n",
    "    with open(os.path.join(spam_path,directory,filename),'rb') as f:\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Martin A posted:\n",
      "Tassos Papadopoulos, the Greek sculptor behind the plan, judged that the\n",
      " limestone of Mount Kerdylio, 70 miles east of Salonika and not far from the\n",
      " Mount Athos monastic community, was ideal for the patriotic sculpture. \n",
      " \n",
      " As well as Alexander's granite features, 240 ft high and 170 ft wide, a\n",
      " museum, a restored amphitheatre and car park for admiring crowds are\n",
      "planned\n",
      "---------------------\n",
      "So is this mountain limestone or granite?\n",
      "If it's limestone, it'll weather pretty fast.\n",
      "\n",
      "------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "4 DVDs Free +s&p Join Now\n",
      "http://us.click.yahoo.com/pt6YBB/NXiEAA/mG3HAA/7gSolB/TM\n",
      "---------------------------------------------------------------------~->\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "forteana-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n"
     ]
    }
   ],
   "source": [
    "# 用上述读取邮件的方法，读取出所有的邮件\n",
    "ham_emails = [load_email(is_spam = False,filename = name) for name in ham_filenames]\n",
    "spam_emails = [load_email(is_spam = True,filename = name) for name in spam_filenames]\n",
    "\n",
    "# 看一个ham示例和一个spam示例，了解数据的外观\n",
    "print(ham_emails[1].get_content().strip())       # get_content() 属于email方法，获取邮件的内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help wanted.  We are a 14 year old fortune 500 company, that is\n",
      "growing at a tremendous rate.  We are looking for individuals who\n",
      "want to work from home.\n",
      "\n",
      "This is an opportunity to make an excellent income.  No experience\n",
      "is required.  We will train you.\n",
      "\n",
      "So if you are looking to be employed from home with a career that has\n",
      "vast opportunities, then go:\n",
      "\n",
      "http://www.basetel.com/wealthnow\n",
      "\n",
      "We are looking for energetic and self motivated people.  If that is you\n",
      "than click on the link and fill out the form, and one of our\n",
      "employement specialist will contact you.\n",
      "\n",
      "To be removed from our link simple go to:\n",
      "\n",
      "http://www.basetel.com/remove.html\n",
      "\n",
      "\n",
      "4139vOLW7-758DoDY1425FRhM1-764SMFc8513fCsLl40\n"
     ]
    }
   ],
   "source": [
    "print(spam_emails[6].get_content().strip()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 电子邮件实际上有很多部分，带有图像和附件（他们可以有自己的附件），查看邮件的各种类型的结构\n",
    "\n",
    "# 获得文本内容的类型\n",
    "def get_email_structure(email):\n",
    "    if isinstance(email,str):\n",
    "        return email\n",
    "    payload = email.get_payload()             # 获取邮件的组成部分\n",
    "    if isinstance(payload,list):              # 若邮件结构复杂，拆分结构\n",
    "        return \"multipart({})\".format(\",\".join([\n",
    "            get_email_structure(sub_email) \n",
    "            for sub_email in payload \n",
    "        ]))\n",
    "    else:\n",
    "        return email.get_content_type()        # 获得文本内容的类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 2, 2: 2, 3: 2, 5: 2, 4: 1})"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter           # 得到列表中各个数出现的次数\n",
    "\n",
    "a = [1,2,3,5,2,3,5,1,4]\n",
    "b = Counter(a)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义一个方法，获得所有邮件中不同文本内容的类型出现的次数\n",
    "from collections import Counter \n",
    "def structures_counter(emails):\n",
    "    structures = Counter()        # structures为字典类型\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] += 1    # 给字典赋值\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 2408),\n",
       " ('multipart(text/plain,application/pgp-signature)', 66),\n",
       " ('multipart(text/plain,text/html)', 8),\n",
       " ('multipart(text/plain,text/plain)', 4),\n",
       " ('multipart(text/plain)', 3),\n",
       " ('multipart(text/plain,application/octet-stream)', 2),\n",
       " ('multipart(text/plain,text/enriched)', 1),\n",
       " ('multipart(text/plain,application/ms-tnef,text/plain)', 1),\n",
       " ('multipart(multipart(text/plain,text/plain,text/plain),application/pgp-signature)',\n",
       "  1),\n",
       " ('multipart(text/plain,video/mng)', 1),\n",
       " ('multipart(text/plain,multipart(text/plain))', 1),\n",
       " ('multipart(text/plain,application/x-pkcs7-signature)', 1),\n",
       " ('multipart(text/plain,multipart(text/plain,text/plain),text/rfc822-headers)',\n",
       "  1),\n",
       " ('multipart(text/plain,multipart(text/plain,text/plain),multipart(multipart(text/plain,application/x-pkcs7-signature)))',\n",
       "  1),\n",
       " ('multipart(text/plain,application/x-java-applet)', 1)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看正常邮件中的类型分布\n",
    "structures_counter(ham_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 218),\n",
       " ('text/html', 183),\n",
       " ('multipart(text/plain,text/html)', 45),\n",
       " ('multipart(text/html)', 20),\n",
       " ('multipart(text/plain)', 19),\n",
       " ('multipart(multipart(text/html))', 5),\n",
       " ('multipart(text/plain,image/jpeg)', 3),\n",
       " ('multipart(text/html,application/octet-stream)', 2),\n",
       " ('multipart(text/plain,application/octet-stream)', 1),\n",
       " ('multipart(text/html,text/plain)', 1),\n",
       " ('multipart(multipart(text/html),application/octet-stream,image/jpeg)', 1),\n",
       " ('multipart(multipart(text/plain,text/html),image/gif)', 1),\n",
       " ('multipart/alternative', 1)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看垃圾邮件中的类型分布,垃圾邮件中HTML格式的文件偏多\n",
    "structures_counter(spam_emails).most_common()         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Return-Path : <12a1mailbot1@web.de>\n",
      "Delivered-To : zzzz@localhost.spamassassin.taint.org\n",
      "Received : from localhost (localhost [127.0.0.1])\tby phobos.labs.spamassassin.taint.org (Postfix) with ESMTP id 136B943C32\tfor <zzzz@localhost>; Thu, 22 Aug 2002 08:17:21 -0400 (EDT)\n",
      "Received : from mail.webnote.net [193.120.211.219]\tby localhost with POP3 (fetchmail-5.9.0)\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 13:17:21 +0100 (IST)\n",
      "Received : from dd_it7 ([210.97.77.167])\tby webnote.net (8.9.3/8.9.3) with ESMTP id NAA04623\tfor <zzzz@spamassassin.taint.org>; Thu, 22 Aug 2002 13:09:41 +0100\n",
      "From : 12a1mailbot1@web.de\n",
      "Received : from r-smtp.korea.com - 203.122.2.197 by dd_it7  with Microsoft SMTPSVC(5.5.1775.675.6);\t Sat, 24 Aug 2002 09:42:10 +0900\n",
      "To : dcek1a1@netsgo.com\n",
      "Subject : Life Insurance - Why Pay More?\n",
      "Date : Wed, 21 Aug 2002 20:31:57 -1600\n",
      "MIME-Version : 1.0\n",
      "Message-ID : <0103c1042001882DD_IT7@dd_it7>\n",
      "Content-Type : text/html; charset=\"iso-8859-1\"\n",
      "Content-Transfer-Encoding : quoted-printable\n"
     ]
    }
   ],
   "source": [
    "# 正常邮件更多的是纯文本，而垃圾邮件有相当多的html\n",
    "# 查看邮件头\n",
    "for header,value in spam_emails[0].items():\n",
    "    print(header,':',value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Life Insurance - Why Pay More?'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_emails[0]['Subject']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拆分训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "X = np.array(ham_emails + spam_emails)\n",
    "y = np.array([0] * len(ham_emails) + [1] * len(spam_emails))       # 将0定为正常邮件，1定为垃圾邮件\n",
    "\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将HTML转换为纯文本\n",
    "    使用Beautifulsoup库，下面的函数首先删除<head>部分，然后将所有<a>标记转换为单词hyperlink,然后去掉所有html标记，只留下纯文本。为了可读性，它还用一个换行符替换多个换行符，最后它取消了HTML实体（例如&gt,或&nbsp）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from html import unescape\n",
    "\n",
    "def html_to_plain_text(html):\n",
    "    text = re.sub('<head.*?>.*?</head>','',html,flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<a.*?>','HYPERLINK',text,flags=re.M | re.S | re.I)    \n",
    "    text = re.sub('<.*?>','',text,flags=re.M | re.S)    \n",
    "    text = re.sub(r'(\\s*\\n)+','\\n',text,flags=re.M | re.S)    \n",
    "    return unescape(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HTML><HEAD><TITLE></TITLE><META http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\"><STYLE>A:link {TEX-DECORATION: none}A:active {TEXT-DECORATION: none}A:visited {TEXT-DECORATION: none}A:hover {COLOR: #0033ff; TEXT-DECORATION: underline}</STYLE><META content=\"MSHTML 6.00.2713.1100\" name=\"GENERATOR\"></HEAD>\n",
      "<BODY text=\"#000000\" vLink=\"#0033ff\" link=\"#0033ff\" bgColor=\"#CCCC99\"><TABLE borderColor=\"#660000\" cellSpacing=\"0\" cellPadding=\"0\" border=\"0\" width=\"100%\"><TR><TD bgColor=\"#CCCC99\" valign=\"top\" colspan=\"2\" height=\"27\">\n",
      "<font size=\"6\" face=\"Arial, Helvetica, sans-serif\" color=\"#660000\">\n",
      "<b>OTC</b></font></TD></TR><TR><TD height=\"2\" bgcolor=\"#6a694f\">\n",
      "<font size=\"5\" face=\"Times New Roman, Times, serif\" color=\"#FFFFFF\">\n",
      "<b>&nbsp;Newsletter</b></font></TD><TD height=\"2\" bgcolor=\"#6a694f\"><div align=\"right\"><font color=\"#FFFFFF\">\n",
      "<b>Discover Tomorrow's Winners&nbsp;</b></font></div></TD></TR><TR><TD height=\"25\" colspan=\"2\" bgcolor=\"#CCCC99\"><table width=\"100%\" border=\"0\"  ...\n"
     ]
    }
   ],
   "source": [
    "# 拿到垃圾邮件中为HTML类型的邮件\n",
    "html_spam_emails = [email for email in X_train[y_train==1] \n",
    "                    if get_email_structure(email) == \"text/html\"]\n",
    "sample_html_spam = html_spam_emails[7]\n",
    "print(sample_html_spam.get_content().strip()[:1000],\"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Watch for analyst \"Strong Buy Recommendations\" and several advisory newsletters picking CBYI.  CBYI has filed to be traded on the OTCBB, share prices historically INCREASE when companies get listed on this larger trading exchange. CBYI is trading around 25 cents and should skyrocket to $2.66 - $3.25 a share in the near future.\n",
      "Put CBYI on your watch list, acquire a position TODAY.\n",
      "REASONS TO INVEST IN CBYI\n",
      "A profitable company and is on track to beat ALL earnings estimates!\n",
      "One of the FASTEST growing distributors in environmental & safety equipment instruments.\n",
      "Excellent management team, several EXCLUSIVE contracts.  IMPRESSIVE client list including the U.S. Air Force, Anheuser-Busch, Chevron Refining and Mitsubishi Heavy Industries, GE-Energy & Environmental Research.\n",
      "RAPIDLY GROWING INDUSTRY\n",
      "Industry revenues exceed $900 million, estimates indicate that there could be as much as $25 bil ...\n"
     ]
    }
   ],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content().strip()[:3000]),\"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 编写一个函数，它以电子邮件为输入对象，不论其格式是什么，都以纯文本形式返回其内容\n",
    "\n",
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\",\"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except:\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\" :\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Wat ...\n"
     ]
    }
   ],
   "source": [
    "print(email_to_text(sample_html_spam)[:100],\"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computations => comput\n",
      "Computation => comput\n",
      "Computing => comput\n",
      "Computed => comput\n",
      "Compute => comput\n",
      "Compulsive => compuls\n"
     ]
    }
   ],
   "source": [
    "# 装自然语言工具包 nltk\n",
    "\n",
    "# 用“url”替换url方法，统计url的个数  urlextract\n",
    "import nltk\n",
    "from  urlextract import URLExtract\n",
    "\n",
    "try:\n",
    "    import nltk\n",
    "\n",
    "    stemmer = nltk.PorterStemmer()              # 提取单词的词根，可将相似的单词归为一类\n",
    "    for word in (\"Computations\", \"Computation\", \"Computing\", \"Computed\", \"Compute\", \"Compulsive\"):\n",
    "        print(word, \"=>\", stemmer.stem(word))\n",
    "except ImportError:\n",
    "    print(\"Error: stemming requires the NLTK module.\")\n",
    "    stemmer = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 前面所做的操作都是数据清洗部分，将所有处理整合到一个转换器中，我们将使用它将电子邮件转换为文字计数器。注意，我们使用python的split()方法将句子拆分为单词，该方法使用空格为分隔符；但例如 汉语和日语脚本通常不在单词之间使用空格， 不过在这个练习中没关系，因为数据集主要是英文的，中文可以使用结巴粉刺来进行拆分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 自定义一个将电子邮件转换为单词计数器的转换器\n",
    "from sklearn.base import BaseEstimator,TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator,TransformerMixin):\n",
    "    def __init__(self,strip_headers=True,lower_case=True,remove_punctuation=True,\n",
    "                 replace_urls=True,replace_numbers=True,stemming=True):\n",
    "        self.strip_headers = strip_headers               # 是否要去掉头部\n",
    "        self.lower_case = lower_case                     # 是否将内容转换成小写\n",
    "        self.remove_punctuation = remove_punctuation     # 是否移除所有的标点符号\n",
    "        self.replace_urls = replace_urls                 # 是否将所有的url进行替换\n",
    "        self.replace_numbers = replace_numbers           # 是否将数字进行替换\n",
    "        self.stemming = stemming                         # 是否提取词根\n",
    "    def fit(self,X,y=None):\n",
    "        return self\n",
    "    def transform(self,X,y=None):\n",
    "        X_transformed = [ ]       # 用于添入转换完的X\n",
    "        # 遍历邮件，将邮件转为文本类型\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"      \n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            if self.replace_urls:\n",
    "                extractor = URLExtract()                              # 初始化URLExtract对象\n",
    "                urls = list(set(extractor.find_urls(text)))          # 在文本中寻找url地址,返回一个列表\n",
    "                urls.sort(key=lambda url: len(url),reverse=True)   # 按照url的长度进行排序\n",
    "                for url in urls:\n",
    "                    text = text.replace(url,\"URL\")                             # 将url地址替换为URL字符\n",
    "            if self.replace_numbers:\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?','NUMBER',text)      # 使用正则将数字替换成NUMBER\n",
    "            if self.remove_punctuation:\n",
    "                text = re.sub(r'\\W+',' ',text,flags=re.M)\n",
    "            word_counts = Counter(text.split())                         # 返回一个字典，对每个单词出现的次数进行统计\n",
    "            if self.stemming and stemmer is not None:                 # 提取词根并且词根不为空\n",
    "                stemmed_word_counts = Counter()\n",
    "                for word,count in word_counts.items():\n",
    "                    stemmed_word = stemmer.stem(word)               # 提取词根\n",
    "                    stemmed_word_counts[stemmed_word] += count      # 对词根重新计数\n",
    "                word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([Counter({'chuck': 1, 'murcko': 1, 'wrote': 1, 'stuff': 1, 'yawn': 1, 'r': 1}),\n",
       "       Counter({'the': 11, 'of': 9, 'and': 8, 'all': 3, 'christian': 3, 'to': 3, 'by': 3, 'jefferson': 2, 'i': 2, 'have': 2, 'superstit': 2, 'one': 2, 'on': 2, 'been': 2, 'ha': 2, 'half': 2, 'rogueri': 2, 'teach': 2, 'jesu': 2, 'some': 1, 'interest': 1, 'quot': 1, 'url': 1, 'thoma': 1, 'examin': 1, 'known': 1, 'word': 1, 'do': 1, 'not': 1, 'find': 1, 'in': 1, 'our': 1, 'particular': 1, 'redeem': 1, 'featur': 1, 'they': 1, 'are': 1, 'alik': 1, 'found': 1, 'fabl': 1, 'mytholog': 1, 'million': 1, 'innoc': 1, 'men': 1, 'women': 1, 'children': 1, 'sinc': 1, 'introduct': 1, 'burnt': 1, 'tortur': 1, 'fine': 1, 'imprison': 1, 'what': 1, 'effect': 1, 'thi': 1, 'coercion': 1, 'make': 1, 'world': 1, 'fool': 1, 'other': 1, 'hypocrit': 1, 'support': 1, 'error': 1, 'over': 1, 'earth': 1, 'six': 1, 'histor': 1, 'american': 1, 'john': 1, 'e': 1, 'remsburg': 1, 'letter': 1, 'william': 1, 'short': 1, 'again': 1, 'becom': 1, 'most': 1, 'pervert': 1, 'system': 1, 'that': 1, 'ever': 1, 'shone': 1, 'man': 1, 'absurd': 1, 'untruth': 1, 'were': 1, 'perpetr': 1, 'upon': 1, 'a': 1, 'larg': 1, 'band': 1, 'dupe': 1, 'import': 1, 'led': 1, 'paul': 1, 'first': 1, 'great': 1, 'corrupt': 1}),\n",
       "       Counter({'url': 4, 's': 3, 'group': 3, 'to': 3, 'in': 2, 'forteana': 2, 'martin': 2, 'an': 2, 'and': 2, 'we': 2, 'is': 2, 'yahoo': 2, 'unsubscrib': 2, 'y': 1, 'adamson': 1, 'wrote': 1, 'for': 1, 'altern': 1, 'rather': 1, 'more': 1, 'factual': 1, 'base': 1, 'rundown': 1, 'on': 1, 'hamza': 1, 'career': 1, 'includ': 1, 'hi': 1, 'belief': 1, 'that': 1, 'all': 1, 'non': 1, 'muslim': 1, 'yemen': 1, 'should': 1, 'be': 1, 'murder': 1, 'outright': 1, 'know': 1, 'how': 1, 'unbias': 1, 'memri': 1, 'don': 1, 't': 1, 'html': 1, 'rob': 1, 'sponsor': 1, 'number': 1, 'dvd': 1, 'free': 1, 'p': 1, 'join': 1, 'now': 1, 'from': 1, 'thi': 1, 'send': 1, 'email': 1, 'egroup': 1, 'com': 1, 'your': 1, 'use': 1, 'of': 1, 'subject': 1})],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 在一些邮件上测试转换器\n",
    "X_few = X_train[:3]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 有了单词计数，我们需要把它们转换成向量.为此，我们将构建另一个转换器，其“fit（）”方法将构建词汇表（最常用单词的有序列表），其“transform（）”方法将使用词汇表将单词计数转换为向量--稀疏矩阵  (压缩的稀疏矩阵，矩阵交给机器学习才能进行学习 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "# 自定义一个转换器，将单词计数转换成向量矩阵\n",
    "class WordCounterToVectorTransformer(BaseEstimator,TransformerMixin):\n",
    "    def __init__(self,vocabulary_size = 1000):\n",
    "        self.vocabulary_size = vocabulary_size        # 词汇量\n",
    "    def fit(self,X,y=None):\n",
    "        total_count = Counter()    # 全部词汇的计数\n",
    "        for word_count in X:      # word_count为每个邮件中每个单词的计数\n",
    "            for word,count in word_count.items():\n",
    "                total_count[word] += min(count,10)      # 设置10次的上限\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]         # most_common 方法使次数从高到低排,截取排在前1000位的单词计数\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index,(word,count) in enumerate(most_common)}\n",
    "        return self\n",
    "    def transform(self,X,y=None):        # transform（）方法统计某一封邮件中的单词在词汇表当中出现的频率统计\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row,word_count in enumerate(X):                        # 对整个数据集进行枚举,row为索引\n",
    "            for word,count in word_count.items():\n",
    "                rows.append(row)                                 # 邮件的索引\n",
    "                cols.append(self.vocabulary_.get(word,0))       # 取得单词在词汇表中的索引位置，0代表未出现在词汇表中\n",
    "                data.append(count)                             # 当前邮件中单词出现的次数\n",
    "        return csr_matrix((data,(rows,cols)),shape = (len(X),self.vocabulary_size +1))      # 输出稀疏矩阵 +1因为第一列要显示未出现在词汇表中的单词统计数\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 0)\t1\n",
      "  (0, 1)\t0\n",
      "  (0, 2)\t1\n",
      "  (1, 0)\t0\n",
      "  (1, 1)\t1\n",
      "  (1, 2)\t1\n",
      "  (2, 0)\t1\n",
      "  (2, 1)\t1\n",
      "  (2, 2)\t0\n",
      "[[1 0 1]\n",
      " [0 1 1]\n",
      " [1 1 0]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 1],\n",
       "       [0, 1, 1],\n",
       "       [1, 1, 0]], dtype=int32)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.sparse import *\n",
    " \n",
    "row =  [0,0,0,1,1,1,2,2,2]      #行指标\n",
    "col =  [0,1,2,0,1,2,0,1,2]     #列指标\n",
    "data = [1,0,1,0,1,1,1,1,0]    #在行指标列指标下的数字\n",
    "team = csr_matrix((data,(row,col)),shape=(3,3))\n",
    "print(team)\n",
    "print(team.todense())         # 压缩的矩阵\n",
    "team.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 11)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=10)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors.shape     # 3封邮件，11个计数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 6,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [99, 11,  9,  8,  3,  1,  3,  1,  3,  2,  3],\n",
       "       [67,  0,  1,  2,  3,  4,  1,  2,  0,  1,  0]], dtype=int32)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few_vectors.toarray()      # 查看完整的矩阵  第3行中 67表示有67个单词没有出现在词汇表当中,0表示the出现0次，2表示and在邮件中出现2次"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'the': 1,\n",
       " 'of': 2,\n",
       " 'and': 3,\n",
       " 'to': 4,\n",
       " 'url': 5,\n",
       " 'all': 6,\n",
       " 'in': 7,\n",
       " 'christian': 8,\n",
       " 'on': 9,\n",
       " 'by': 10}"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer.vocabulary_         # 访问词汇表"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 我们现在准备训练我们的第一个垃圾邮件分类器！让我们转换整个数据集："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据转换流水线\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_to_wordcount\", EmailToWordCounterTransformer()),         # 将电子邮件转换为单词计数器\n",
    "    (\"wordcount_to_vector\", WordCounterToVectorTransformer()),       # 将单词计数转换成向量矩阵\n",
    "])\n",
    "\n",
    "# 得到被处理过的训练集\n",
    "X_train_transformed = preprocess_pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  ................................................................\n",
      "[CV] .................................... , score=0.981, total=   0.2s\n",
      "[CV]  ................................................................\n",
      "[CV] .................................... , score=0.983, total=   0.1s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .................................... , score=0.993, total=   0.3s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\svm\\base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.6s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9854166666666667"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 逻辑回归分类器\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)                  # 初始化逻辑回归分类器\n",
    "score = cross_val_score(log_clf, X_train_transformed, y_train, cv=3, verbose=3)   # 使用交叉验证进行评估\n",
    "score.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 得到分数超过98%，可以尝试多个模型，选择最好的模型，并使用交叉验证对它们进行微调。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "精度: 93.94%\n",
      "召回: 97.89%\n"
     ]
    }
   ],
   "source": [
    "# 在测试集上得到的精度/召回率\n",
    "\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)     # 对测试集进行转换\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
    "log_clf.fit(X_train_transformed, y_train)           # 用训练集进行模型的训练\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)        # 对转换过的测试集进行预测\n",
    "\n",
    "print(\"精度: {:.2f}%\".format(100 * precision_score(y_test, y_pred)))\n",
    "print(\"召回: {:.2f}%\".format(100 * recall_score(y_test, y_pred))) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 流程总结\n",
    "    1.加载数据并纵观全局\n",
    "    2.获取邮件的组成结构\n",
    "    3.对邮件结构类型进行分析，发现垃圾邮件大都有HTML结构\n",
    "    4.数据清洗，定义email对象中的HTML转换成纯文本方法\n",
    "    5.对数据集拆分为训练集和测试集\n",
    "    6.数据处理转换，对邮件的文本内容进行分词处理，通过nltk进行词干提取，对邮件中出现的词汇进行计数统计，对所有邮件统计出了一个词汇表\n",
    "    7.通过词汇表和邮件单词计数统计，，将单词计数转换成向量矩阵\n",
    "    8.把数据清洗和数据处理封装成两个转换器\n",
    "    9.通过流水线来自动化处理数据\n",
    "    10.通过逻辑回归线性分类器进行模型训练\n",
    "    11.使用交叉验证进行微调\n",
    "    12.在测试集上得到精度和召回\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
